<!DOCTYPE html><html lang="zh-CN" data-theme="light"><head><meta charset="UTF-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no"><title>大数据之数据仓库Hive05_企业级调优 | Oct25-X</title><meta name="keywords" content="Hive"><meta name="author" content="Oct25-X"><meta name="copyright" content="Oct25-X"><meta name="format-detection" content="telephone=no"><meta name="theme-color" content="#ffffff"><meta name="description" content="计算资源配置一、Yarn 资源配置 nodemanager 分配给 container 的内存  1234&lt;property&gt;    &lt;name&gt;yarn.nodemanager.resource.memory-mb&lt;&#x2F;name&gt;    &lt;value&gt;65536&lt;&#x2F;value&gt; &lt;!--64G--&gt;&lt;&#x2F;property&amp;">
<meta property="og:type" content="article">
<meta property="og:title" content="大数据之数据仓库Hive05_企业级调优">
<meta property="og:url" content="https://oct25-x.atomgit.net/oct25-xxxxx/2023/07/23/17%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8B%E6%95%B0%E6%8D%AE%E4%BB%93%E5%BA%93Hive05_%E4%BC%81%E4%B8%9A%E7%BA%A7%E8%B0%83%E4%BC%98/index.html">
<meta property="og:site_name" content="Oct25-X">
<meta property="og:description" content="计算资源配置一、Yarn 资源配置 nodemanager 分配给 container 的内存  1234&lt;property&gt;    &lt;name&gt;yarn.nodemanager.resource.memory-mb&lt;&#x2F;name&gt;    &lt;value&gt;65536&lt;&#x2F;value&gt; &lt;!--64G--&gt;&lt;&#x2F;property&amp;">
<meta property="og:locale" content="zh_CN">
<meta property="og:image" content="https://oct25-x.atomgit.net/oct25-xxxxx/img/cover_img/ctt17.png">
<meta property="article:published_time" content="2023-07-23T10:21:16.000Z">
<meta property="article:modified_time" content="2024-02-17T14:17:55.403Z">
<meta property="article:author" content="Oct25-X">
<meta property="article:tag" content="Hive">
<meta name="twitter:card" content="summary">
<meta name="twitter:image" content="https://oct25-x.atomgit.net/oct25-xxxxx/img/cover_img/ctt17.png"><link rel="shortcut icon" href="/oct25-xxxxx/img/logo.jpg"><link rel="canonical" href="https://oct25-x.atomgit.net/oct25-xxxxx/2023/07/23/17%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8B%E6%95%B0%E6%8D%AE%E4%BB%93%E5%BA%93Hive05_%E4%BC%81%E4%B8%9A%E7%BA%A7%E8%B0%83%E4%BC%98/"><link rel="preconnect" href="//cdn.jsdelivr.net"/><link rel="preconnect" href="//busuanzi.ibruce.info"/><link rel="stylesheet" href="/oct25-xxxxx/css/index.css"><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@fortawesome/fontawesome-free@6/css/all.min.css" media="print" onload="this.media='all'"><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@fancyapps/ui/dist/fancybox.css" media="print" onload="this.media='all'"><script>const GLOBAL_CONFIG = { 
  root: '/oct25-xxxxx/',
  algolia: undefined,
  localSearch: {"path":"search.json","languages":{"hits_empty":"找不到您查询的内容：${query}"}},
  translate: {"defaultEncoding":2,"translateDelay":0,"msgToTraditionalChinese":"繁","msgToSimplifiedChinese":"簡"},
  noticeOutdate: undefined,
  highlight: {"plugin":"highlighjs","highlightCopy":true,"highlightLang":true,"highlightHeightLimit":false},
  copy: {
    success: '复制成功',
    error: '复制错误',
    noSupport: '浏览器不支持'
  },
  relativeDate: {
    homepage: false,
    post: false
  },
  runtime: '',
  date_suffix: {
    just: '刚刚',
    min: '分钟前',
    hour: '小时前',
    day: '天前',
    month: '个月前'
  },
  copyright: undefined,
  lightbox: 'fancybox',
  Snackbar: undefined,
  source: {
    justifiedGallery: {
      js: 'https://cdn.jsdelivr.net/npm/flickr-justified-gallery@2/dist/fjGallery.min.js',
      css: 'https://cdn.jsdelivr.net/npm/flickr-justified-gallery@2/dist/fjGallery.min.css'
    }
  },
  isPhotoFigcaption: false,
  islazyload: false,
  isAnchor: false
}</script><script id="config-diff">var GLOBAL_CONFIG_SITE = {
  title: '大数据之数据仓库Hive05_企业级调优',
  isPost: true,
  isHome: false,
  isHighlightShrink: false,
  isToc: true,
  postUpdate: '2024-02-17 22:17:55'
}</script><noscript><style type="text/css">
  #nav {
    opacity: 1
  }
  .justified-gallery img {
    opacity: 1
  }

  #recent-posts time,
  #post-meta time {
    display: inline !important
  }
</style></noscript><script>(win=>{
    win.saveToLocal = {
      set: function setWithExpiry(key, value, ttl) {
        if (ttl === 0) return
        const now = new Date()
        const expiryDay = ttl * 86400000
        const item = {
          value: value,
          expiry: now.getTime() + expiryDay,
        }
        localStorage.setItem(key, JSON.stringify(item))
      },

      get: function getWithExpiry(key) {
        const itemStr = localStorage.getItem(key)

        if (!itemStr) {
          return undefined
        }
        const item = JSON.parse(itemStr)
        const now = new Date()

        if (now.getTime() > item.expiry) {
          localStorage.removeItem(key)
          return undefined
        }
        return item.value
      }
    }
  
    win.getScript = url => new Promise((resolve, reject) => {
      const script = document.createElement('script')
      script.src = url
      script.async = true
      script.onerror = reject
      script.onload = script.onreadystatechange = function() {
        const loadState = this.readyState
        if (loadState && loadState !== 'loaded' && loadState !== 'complete') return
        script.onload = script.onreadystatechange = null
        resolve()
      }
      document.head.appendChild(script)
    })
  
      win.activateDarkMode = function () {
        document.documentElement.setAttribute('data-theme', 'dark')
        if (document.querySelector('meta[name="theme-color"]') !== null) {
          document.querySelector('meta[name="theme-color"]').setAttribute('content', '#0d0d0d')
        }
      }
      win.activateLightMode = function () {
        document.documentElement.setAttribute('data-theme', 'light')
        if (document.querySelector('meta[name="theme-color"]') !== null) {
          document.querySelector('meta[name="theme-color"]').setAttribute('content', '#ffffff')
        }
      }
      const t = saveToLocal.get('theme')
    
          if (t === 'dark') activateDarkMode()
          else if (t === 'light') activateLightMode()
        
      const asideStatus = saveToLocal.get('aside-status')
      if (asideStatus !== undefined) {
        if (asideStatus === 'hide') {
          document.documentElement.classList.add('hide-aside')
        } else {
          document.documentElement.classList.remove('hide-aside')
        }
      }
    
    const detectApple = () => {
      if(/iPad|iPhone|iPod|Macintosh/.test(navigator.userAgent)){
        document.documentElement.classList.add('apple')
      }
    }
    detectApple()
    })(window)</script><link rel="stylesheet" href="/css/rightMenu.css"><meta name="generator" content="Hexo 6.1.0"></head><body><div id="sidebar"><div id="menu-mask"></div><div id="sidebar-menus"><div class="avatar-img is-center"><img src="/oct25-xxxxx/img/avatar.jpg" onerror="onerror=null;src='/img/friend_404.gif'" alt="avatar"/></div><div class="site-data is-center"><div class="data-item"><a href="/oct25-xxxxx/archives/"><div class="headline">文章</div><div class="length-num">52</div></a></div><div class="data-item"><a href="/oct25-xxxxx/tags/"><div class="headline">标签</div><div class="length-num">17</div></a></div><div class="data-item"><a href="/oct25-xxxxx/categories/"><div class="headline">分类</div><div class="length-num">8</div></a></div></div><hr/><div class="menus_items"><div class="menus_item"><a class="site-page" href="/oct25-xxxxx/"><i class="fa-fw fas fa-home"></i><span> 首页</span></a></div><div class="menus_item"><a class="site-page" href="/oct25-xxxxx/archives/"><i class="fa-fw fas fa-archive"></i><span> 归档</span></a></div><div class="menus_item"><a class="site-page" href="/oct25-xxxxx/tags/"><i class="fa-fw fas fa-tags"></i><span> 标签</span></a></div><div class="menus_item"><a class="site-page" href="/oct25-xxxxx/categories/"><i class="fa-fw fas fa-folder-open"></i><span> 分类</span></a></div><div class="menus_item"><a class="site-page" href="/oct25-xxxxx/about/"><i class="fa-fw fas fa-heart"></i><span> 关于</span></a></div></div></div></div><div class="post" id="body-wrap"><header class="post-bg" id="page-header" style="background-image: url('/oct25-xxxxx/img/top_img/top_bg16.jpg')"><nav id="nav"><span id="blog_name"><a id="site-name" href="/oct25-xxxxx/">Oct25-X</a></span><div id="menus"><div id="search-button"><a class="site-page social-icon search"><i class="fas fa-search fa-fw"></i><span> 搜索</span></a></div><div class="menus_items"><div class="menus_item"><a class="site-page" href="/oct25-xxxxx/"><i class="fa-fw fas fa-home"></i><span> 首页</span></a></div><div class="menus_item"><a class="site-page" href="/oct25-xxxxx/archives/"><i class="fa-fw fas fa-archive"></i><span> 归档</span></a></div><div class="menus_item"><a class="site-page" href="/oct25-xxxxx/tags/"><i class="fa-fw fas fa-tags"></i><span> 标签</span></a></div><div class="menus_item"><a class="site-page" href="/oct25-xxxxx/categories/"><i class="fa-fw fas fa-folder-open"></i><span> 分类</span></a></div><div class="menus_item"><a class="site-page" href="/oct25-xxxxx/about/"><i class="fa-fw fas fa-heart"></i><span> 关于</span></a></div></div><div id="toggle-menu"><a class="site-page"><i class="fas fa-bars fa-fw"></i></a></div></div></nav><div id="post-info"><h1 class="post-title">大数据之数据仓库Hive05_企业级调优</h1><div id="post-meta"><div class="meta-firstline"><span class="post-meta-date"><i class="far fa-calendar-alt fa-fw post-meta-icon"></i><span class="post-meta-label">发表于</span><time class="post-meta-date-created" datetime="2023-07-23T10:21:16.000Z" title="发表于 2023-07-23 18:21:16">2023-07-23</time><span class="post-meta-separator">|</span><i class="fas fa-history fa-fw post-meta-icon"></i><span class="post-meta-label">更新于</span><time class="post-meta-date-updated" datetime="2024-02-17T14:17:55.403Z" title="更新于 2024-02-17 22:17:55">2024-02-17</time></span><span class="post-meta-categories"><span class="post-meta-separator">|</span><i class="fas fa-inbox fa-fw post-meta-icon"></i><a class="post-meta-categories" href="/oct25-xxxxx/categories/%E5%A4%A7%E6%95%B0%E6%8D%AE%E7%BB%84%E4%BB%B6/">大数据组件</a></span></div><div class="meta-secondline"><span class="post-meta-separator">|</span><span class="post-meta-pv-cv" id="" data-flag-title="大数据之数据仓库Hive05_企业级调优"><i class="far fa-eye fa-fw post-meta-icon"></i><span class="post-meta-label">阅读量:</span><span id="busuanzi_value_page_pv"></span></span></div></div></div></header><main class="layout" id="content-inner"><div id="post"><article class="post-content" id="article-container"><h1 id="计算资源配置"><a href="#计算资源配置" class="headerlink" title="计算资源配置"></a>计算资源配置</h1><h2 id="一、Yarn-资源配置"><a href="#一、Yarn-资源配置" class="headerlink" title="一、Yarn 资源配置"></a>一、Yarn 资源配置</h2><ol>
<li>nodemanager 分配给 container 的内存</li>
</ol>
<figure class="highlight xml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line"><span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">name</span>&gt;</span>yarn.nodemanager.resource.memory-mb<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">value</span>&gt;</span>65536<span class="tag">&lt;/<span class="name">value</span>&gt;</span> <span class="comment">&lt;!--64G--&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br></pre></td></tr></table></figure>

<ol start="2">
<li>nodemanager 分配给 container 的 CPU 核数</li>
</ol>
<figure class="highlight xml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line"><span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">name</span>&gt;</span>yarn.nodemanager.resource.cpu-vcores<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">value</span>&gt;</span>16<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br></pre></td></tr></table></figure>

<ol start="3">
<li>单个 container 能够使用的最大内存</li>
</ol>
<figure class="highlight xml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line"><span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">name</span>&gt;</span>yarn.scheduler.maximum-allocation-mb<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">value</span>&gt;</span>16384<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br></pre></td></tr></table></figure>

<ol start="4">
<li>单个 container 能够使用的最小内存</li>
</ol>
<figure class="highlight xml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line"><span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">name</span>&gt;</span>yarn.scheduler.minimum-allocation-mb<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">value</span>&gt;</span>512<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br></pre></td></tr></table></figure>

<h2 id="二、MapReduce-资源配置"><a href="#二、MapReduce-资源配置" class="headerlink" title="二、MapReduce 资源配置"></a>二、MapReduce 资源配置</h2><ol>
<li>单个 maptask 申请的 container 容器内存大小，默认1024</li>
</ol>
<figure class="highlight sql"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">set</span> mapreduce.map.memory.mb<span class="operator">=</span><span class="number">2048</span>;</span><br></pre></td></tr></table></figure>

<ol start="2">
<li>单个 maptask 申请的 container 容器 cpu 核数，默认1</li>
</ol>
<figure class="highlight xml"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">set mapreduce.map.cpu.vcores=1;</span><br></pre></td></tr></table></figure>

<ol start="3">
<li>单个 reducetask 申请的 container 容器内存大小，默认1024</li>
</ol>
<figure class="highlight xml"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">set  mapreduce.reduce.memory.mb=2048;</span><br></pre></td></tr></table></figure>

<ol start="4">
<li>单个 reducetask 申请的 container 容器 cpu 核数，默认1</li>
</ol>
<figure class="highlight xml"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">set mapreduce.reduce.cpu.vcores=1;</span><br></pre></td></tr></table></figure>

<h1 id="Explain-执行计划"><a href="#Explain-执行计划" class="headerlink" title="Explain 执行计划"></a>Explain 执行计划</h1><h2 id="一、Explain-概述"><a href="#一、Explain-概述" class="headerlink" title="一、Explain 概述"></a>一、Explain 概述</h2><p>Explain 呈现的执行计划，由一系列 Stage 组成，这一系列 Stage 具有依赖关系，每个 Stage 对应一个 MapReduce Job，或者一个文件系统操作等。</p>
<p>若某个 Stage 对应的一个 MapReduce Job，其 Map 端和 Reduce 端的计算逻辑分别由 Map Operator Tree 和 Reduce Operator Tree 进行描述。</p>
<p>Operator Tree 由一系列的 Operator 组成，一个 Operator 代表在 Map 或 Reduce 阶段的一个单一的逻辑操作，例如 TableScan Operator，Select Operator，Join Operator 等。</p>
<p><img src="/oct25-xxxxx/img/hive/06_explain.png" alt="explain 执行计划"></p>
<p>常见的 Operator 及其作用如下：</p>
<ul>
<li><p>TableScan：表扫描操作，通常map端第一个操作肯定是表扫描操作</p>
</li>
<li><p>Select Operator：选取操作</p>
</li>
<li><p>Group By Operator：分组聚合操作</p>
</li>
<li><p>Reduce Output Operator：输出到 reduce 操作</p>
</li>
<li><p>Filter Operator：过滤操作</p>
</li>
<li><p>Join Operator：join 操作</p>
</li>
<li><p>File Output Operator：文件输出操作</p>
</li>
<li><p>Fetch Operator 客户端获取数据操作</p>
</li>
</ul>
<h2 id="二、基本语法"><a href="#二、基本语法" class="headerlink" title="二、基本语法"></a>二、基本语法</h2><figure class="highlight sql"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">explain [formatted <span class="operator">|</span> extended <span class="operator">|</span> depenency] query<span class="operator">-</span><span class="keyword">sql</span></span><br></pre></td></tr></table></figure>

<p>注：FORMATTED、EXTENDED、DEPENDENCY关键字为可选项，各自作用如下。</p>
<ul>
<li><p>FORMATTED：将执行计划以JSON字符串的形式输出</p>
</li>
<li><p>EXTENDED：输出执行计划中的额外信息，通常是读写的文件名等信息</p>
</li>
<li><p>DEPENDENCY：输出执行计划读取的表及分区</p>
</li>
</ul>
<h2 id="三、示例"><a href="#三、示例" class="headerlink" title="三、示例"></a>三、示例</h2><figure class="highlight sql"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">explain</span><br><span class="line"><span class="keyword">select</span> user_id, <span class="built_in">count</span>(<span class="operator">*</span>)</span><br><span class="line"><span class="keyword">from</span> order_detail</span><br><span class="line"><span class="keyword">group</span> <span class="keyword">by</span> user_id;</span><br></pre></td></tr></table></figure>

<p><img src="/oct25-xxxxx/img/hive/06_explain.png" alt="explain 执行计划"></p>
<h1 id="HQL-语法优化之分组聚合优化"><a href="#HQL-语法优化之分组聚合优化" class="headerlink" title="HQL 语法优化之分组聚合优化"></a>HQL 语法优化之分组聚合优化</h1><h2 id="一、优化说明"><a href="#一、优化说明" class="headerlink" title="一、优化说明"></a>一、优化说明</h2><p>Hive 中未经优化的分组聚合，是通过一个 MapReduce Job 实现的。Map 端负责读取数据，并按照分组字段分区，通过Shuffle，将数据发往 Reduce 端，各组数据在 Reduce 端完成最终的聚合运算。</p>
<p>Hive 对分组聚合的优化主要围绕着减少 Shuffle 数据量进行，具体做法是 map-side 聚合。所谓 map-side 聚合，就是在 map 端维护一个 hash table，利用其完成部分的聚合，然后将部分聚合的结果，按照分组字段分区，发送至 reduce端，完成最终的聚合。map-side 聚合能有效减少 shuffle 的数据量，提高分组聚合运算的效率。</p>
<figure class="highlight sql"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">--启用map-side聚合</span></span><br><span class="line"><span class="keyword">set</span> hive.map.aggr<span class="operator">=</span><span class="literal">true</span>;</span><br><span class="line"></span><br><span class="line"><span class="comment">--用于检测源表数据是否适合进行map-side聚合。检测的方法是：先对若干条数据进行map-side聚合，若聚合后的条数和聚合前的条数比值小于该值，则认为该表适合进行map-side聚合；否则，认为该表数据不适合进行map-side聚合，后续数据便不再进行map-side聚合。</span></span><br><span class="line"><span class="keyword">set</span> hive.map.aggr.hash.min.reduction<span class="operator">=</span><span class="number">0.5</span>;</span><br><span class="line"></span><br><span class="line"><span class="comment">--用于检测源表是否适合map-side聚合的条数。</span></span><br><span class="line"><span class="keyword">set</span> hive.groupby.mapaggr.checkinterval<span class="operator">=</span><span class="number">100000</span>;</span><br><span class="line"></span><br><span class="line"><span class="comment">--map-side聚合所用的hash table，占用map task堆内存的最大比例，若超出该值，则会对hash table进行一次flush。</span></span><br><span class="line"><span class="keyword">set</span> hive.map.aggr.hash.force.flush.memory.threshold<span class="operator">=</span><span class="number">0.9</span>;</span><br></pre></td></tr></table></figure>

<h2 id="二、优化示例"><a href="#二、优化示例" class="headerlink" title="二、优化示例"></a>二、优化示例</h2><figure class="highlight sql"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">select</span> product_id, <span class="built_in">count</span>(<span class="operator">*</span>) <span class="keyword">from</span> order_detail <span class="keyword">group</span> <span class="keyword">by</span> product_id;</span><br></pre></td></tr></table></figure>

<h3 id="优化前执行计划"><a href="#优化前执行计划" class="headerlink" title="优化前执行计划"></a>优化前执行计划</h3><p>优化前，执行计划如下图所示：</p>
<p><img src="/oct25-xxxxx/img/hive/07_%E5%88%86%E7%BB%84%E8%81%9A%E5%90%88%E4%BC%98%E5%8C%96%E5%89%8Dexplain.png" alt="分组聚合优化前explain"></p>
<h3 id="优化后执行计划"><a href="#优化后执行计划" class="headerlink" title="优化后执行计划"></a>优化后执行计划</h3><p>优化后，执行计划如下图所示：</p>
<p><img src="/oct25-xxxxx/img/hive/08_%E5%88%86%E7%BB%84%E8%81%9A%E5%90%88%E4%BC%98%E5%8C%96%E5%90%8Eexplain.png" alt="分组聚合优化后explain"></p>
<h1 id="HQL-语法优化之-Join-优化"><a href="#HQL-语法优化之-Join-优化" class="headerlink" title="HQL 语法优化之 Join 优化"></a>HQL 语法优化之 Join 优化</h1><h2 id="一、join-算法说明"><a href="#一、join-算法说明" class="headerlink" title="一、join 算法说明"></a>一、join 算法说明</h2><p>Hive 拥有多种 join 算法，包括 Common Join，Map Join，Bucket Map Join，Sort Merge Buckt Map Join 等，下面对每种 join 算法做简要说明：</p>
<h3 id="Common-Join"><a href="#Common-Join" class="headerlink" title="Common Join"></a>Common Join</h3><p>Common Join 是 Hiv e中最稳定的 join 算法，其通过一个 MapReduce Job 完成一个 join 操作。</p>
<p>Map 端负责读取 join 操作所需表的数据，并按照关联字段进行分区，通过 Shuffle，将其发送到 Reduce 端，相同 key的数据在 Reduce 端完成最终的 Join 操作。</p>
<h3 id="Map-Join"><a href="#Map-Join" class="headerlink" title="Map Join"></a>Map Join</h3><p>Map Join 算法可以通过两个只有 map 阶段的 Job 完成一个 join 操作。其适用场景为大表 join 小表。若某 join 操作满足要求，则第一个 Job 会读取小表数据，将其制作为 hash table，并上传至 Hadoop 分布式缓存（本质上是上传至 HDFS）。第二个 Job 会先从分布式缓存中读取小表数据，并缓存在 Map Task 的内存中，然后扫描大表数据，这样在map 端即可完成关联操作。</p>
<h3 id="Bucket-Map-Join"><a href="#Bucket-Map-Join" class="headerlink" title="Bucket Map Join"></a>Bucket Map Join</h3><p>Bucket Map Join 是对 Map Join 算法的改进，其打破了 Map Join 只适用于大表 join 小表的限制，可用于大表 join 大表的场景。</p>
<p>Bucket Map Join 的核心思想是：若能保证参与 join 的表均为分桶表，且关联字段为分桶字段，且其中一张表的分桶数量是另外一张表分桶数量的整数倍，就能保证参与join的两张表的分桶之间具有明确的关联关系，所以就可以在两表的分桶间进行 Map Join 操作了。这样一来，第二个Job的Map端就无需再缓存小表的全表数据了，而只需缓存其所需的分桶即可。</p>
<h3 id="Sort-Merge-Bucket-Map-Join"><a href="#Sort-Merge-Bucket-Map-Join" class="headerlink" title="Sort Merge Bucket Map Join"></a>Sort Merge Bucket Map Join</h3><p>Sort Merge Bucket Map Join（简称SMB Map Join）基于 Bucket Map Join。SMB Map Join 要求，参与 join 的表均为分桶表，且需保证分桶内的数据是有序的，且分桶字段、排序字段和关联字段为相同字段，且其中一张表的分桶数量是另外一张表分桶数量的整数倍。</p>
<h2 id="二、Map-Join"><a href="#二、Map-Join" class="headerlink" title="二、Map Join"></a>二、Map Join</h2><h3 id="优化参数"><a href="#优化参数" class="headerlink" title="优化参数"></a>优化参数</h3><figure class="highlight sql"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre></td><td class="code"><pre><span class="line"><span class="operator">!</span>[<span class="number">09</span>_mapjoin优化前explain](D:\其他\笔记资料<span class="number">3</span>\BigData03_数据仓库Hive\img\<span class="number">09</span>_mapjoin优化前explain.png)<span class="comment">--启动Map Join自动转换</span></span><br><span class="line"><span class="keyword">set</span> hive.auto.convert.join<span class="operator">=</span><span class="literal">true</span>;</span><br><span class="line"></span><br><span class="line"><span class="comment">--一个Common Join operator转为Map Join operator的判断条件,若该Common Join相关的表中,存在n-1张表的已知大小总和&lt;=该值,则生成一个Map Join计划,此时可能存在多种n-1张表的组合均满足该条件,则hive会为每种满足条件的组合均生成一个Map Join计划,同时还会保留原有的Common Join计划作为后备(back up)计划,实际运行时,优先执行Map Join计划，若不能执行成功，则启动Common Join后备计划。</span></span><br><span class="line"><span class="keyword">set</span> hive.mapjoin.smalltable.filesize<span class="operator">=</span><span class="number">250000</span>;</span><br><span class="line"></span><br><span class="line"><span class="comment">--开启无条件转Map Join</span></span><br><span class="line"><span class="keyword">set</span> hive.auto.convert.join.noconditionaltask<span class="operator">=</span><span class="literal">true</span>;</span><br><span class="line"></span><br><span class="line"><span class="comment">--无条件转Map Join时的小表之和阈值,若一个Common Join operator相关的表中，存在n-1张表的大小总和&lt;=该值,此时hive便不会再为每种n-1张表的组合均生成Map Join计划,同时也不会保留Common Join作为后备计划。而是只生成一个最优的Map Join计划。</span></span><br><span class="line"><span class="keyword">set</span> hive.auto.convert.join.noconditionaltask.size<span class="operator">=</span><span class="number">10000000</span>;</span><br></pre></td></tr></table></figure>

<h3 id="优化示例"><a href="#优化示例" class="headerlink" title="优化示例"></a>优化示例</h3><figure class="highlight sql"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">select</span> <span class="operator">*</span></span><br><span class="line"><span class="keyword">from</span> order_detail od</span><br><span class="line"><span class="keyword">join</span> product_info product <span class="keyword">on</span> od.product_id <span class="operator">=</span> product.id</span><br><span class="line"><span class="keyword">join</span> province_info province <span class="keyword">on</span> od.province_id <span class="operator">=</span> province.id;</span><br></pre></td></tr></table></figure>

<h4 id="优化前执行计划-1"><a href="#优化前执行计划-1" class="headerlink" title="优化前执行计划"></a>优化前执行计划</h4><p>优化前执行计划如下图所示：</p>
<p><img src="/oct25-xxxxx/img/hive/09_mapjoin%E4%BC%98%E5%8C%96%E5%89%8Dexplain.png" alt="mapjoin优化前explain"></p>
<h4 id="方案一优化后执行计划"><a href="#方案一优化后执行计划" class="headerlink" title="方案一优化后执行计划"></a>方案一优化后执行计划</h4><figure class="highlight sql"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">--启用Map Join自动转换</span></span><br><span class="line"><span class="keyword">set</span> hive.auto.convert.join<span class="operator">=</span><span class="literal">true</span>;</span><br><span class="line"><span class="comment">--不使用无条件转Map Join</span></span><br><span class="line"><span class="keyword">set</span> hive.auto.convert.join.noconditionaltask<span class="operator">=</span><span class="literal">false</span>;</span><br><span class="line"><span class="comment">--调整下面参数，使其大于等于product_info</span></span><br><span class="line"><span class="keyword">set</span> hive.mapjoin.smalltable.filesize<span class="operator">=</span><span class="number">25285707</span>;</span><br></pre></td></tr></table></figure>

<p>调整后执行计划如下图所示：</p>
<p><img src="/oct25-xxxxx/img/hive/10_mapjoin%E4%BC%98%E5%8C%96%E5%89%8Dexplain.png" alt="mapjoin优化前explain"></p>
<h4 id="方案二优化后执行计划"><a href="#方案二优化后执行计划" class="headerlink" title="方案二优化后执行计划"></a>方案二优化后执行计划</h4><figure class="highlight sql"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">--启用Map Join自动转换。</span></span><br><span class="line"><span class="keyword">set</span> hive.auto.convert.join<span class="operator">=</span><span class="literal">true</span>;</span><br><span class="line"><span class="comment">--使用无条件转Map Join。</span></span><br><span class="line"><span class="keyword">set</span> hive.auto.convert.join.noconditionaltask<span class="operator">=</span><span class="literal">true</span>;</span><br><span class="line"><span class="comment">--调整下面参数，使其大于等于product_info和province_info之和。</span></span><br><span class="line"><span class="keyword">set</span> hive.auto.convert.join.noconditionaltask.size<span class="operator">=</span><span class="number">25286076</span>;</span><br></pre></td></tr></table></figure>

<p>调整后执行计划如下图所示：</p>
<p><img src="/oct25-xxxxx/img/hive/10_mapjoin%E4%BC%98%E5%8C%96%E5%89%8Dexplain2.png" alt="mapjoin优化前explain"></p>
<h1 id="HQL-语法优化之数据倾斜"><a href="#HQL-语法优化之数据倾斜" class="headerlink" title="HQL 语法优化之数据倾斜"></a>HQL 语法优化之数据倾斜</h1><h2 id="一、数据倾斜概述"><a href="#一、数据倾斜概述" class="headerlink" title="一、数据倾斜概述"></a>一、数据倾斜概述</h2><p>数据倾斜问题，通常是指参与计算的数据分布不均，即某个key或者某些key的数据量远超其他 key，导致在 shuffle 阶段，大量相同 key 的数据被发往同一个 Reduce，进而导致该 Reduce 所需的时间远超其他 Reduce，成为整个任务的瓶颈。</p>
<p>Hive 中的数据倾斜常出现在分组聚合和 join 操作的场景中，下面分别介绍在上述两种场景下的优化思路。</p>
<h2 id="二、分组聚合导致数据倾斜"><a href="#二、分组聚合导致数据倾斜" class="headerlink" title="二、分组聚合导致数据倾斜"></a>二、分组聚合导致数据倾斜</h2><h3 id="Map-Side-聚合"><a href="#Map-Side-聚合" class="headerlink" title="Map-Side 聚合"></a>Map-Side 聚合</h3><p>开启 Map-Side 聚合后，数据会现在 Map 端完成部分聚合工作。这样一来即便原始数据是倾斜的，经过 Map 端的初步聚合后，发往 Reduce 的数据也就不再倾斜了。最佳状态下，Map 端聚合能完全屏蔽数据倾斜问题。</p>
<figure class="highlight sql"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">--启用map-side聚合</span></span><br><span class="line"><span class="keyword">set</span> hive.map.aggr<span class="operator">=</span><span class="literal">true</span>;</span><br><span class="line"></span><br><span class="line"><span class="comment">--用于检测源表数据是否适合进行map-side聚合。检测的方法是：先对若干条数据进行map-side聚合，若聚合后的条数和聚合前的条数比值小于该值，则认为该表适合进行map-side聚合；否则，认为该表数据不适合进行map-side聚合，后续数据便不再进行map-side聚合。</span></span><br><span class="line"><span class="keyword">set</span> hive.map.aggr.hash.min.reduction<span class="operator">=</span><span class="number">0.5</span>;</span><br><span class="line"></span><br><span class="line"><span class="comment">--用于检测源表是否适合map-side聚合的条数。</span></span><br><span class="line"><span class="keyword">set</span> hive.groupby.mapaggr.checkinterval<span class="operator">=</span><span class="number">100000</span>;</span><br><span class="line"></span><br><span class="line"><span class="comment">--map-side聚合所用的hash table，占用map task堆内存的最大比例，若超出该值，则会对hash table进行一次flush。</span></span><br><span class="line"><span class="keyword">set</span> hive.map.aggr.hash.force.flush.memory.threshold<span class="operator">=</span><span class="number">0.9</span>;</span><br></pre></td></tr></table></figure>

<h3 id="Skew-GroupBy-优化"><a href="#Skew-GroupBy-优化" class="headerlink" title="Skew-GroupBy 优化"></a>Skew-GroupBy 优化</h3><p>Skew-GroupBy 的原理是启动两个 MR 任务，第一个 MR 按照随机数分区，将数据分散发送到 Reduce，完成部分聚合，第二个MR按照分组字段分区，完成最终聚合。</p>
<figure class="highlight sql"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">--启用分组聚合数据倾斜优化</span></span><br><span class="line"><span class="keyword">set</span> hive.groupby.skewindata<span class="operator">=</span><span class="literal">true</span>;</span><br></pre></td></tr></table></figure>

<h1 id="HQL-语法优化之小文件合并"><a href="#HQL-语法优化之小文件合并" class="headerlink" title="HQL 语法优化之小文件合并"></a>HQL 语法优化之小文件合并</h1><p>小文件合并优化，分为两个方面，分别是 Map 端输入的小文件合并，和 Reduce 端输出的小文件合并。</p>
<h2 id="一、Map-端输入文件合并"><a href="#一、Map-端输入文件合并" class="headerlink" title="一、Map 端输入文件合并"></a>一、Map 端输入文件合并</h2><p>合并 Map 端输入的小文件，是指将多个小文件划分到一个切片中，进而由一个 Map Task 去处理。目的是防止为单个小文件启动一个 Map Task，浪费计算资源。</p>
<figure class="highlight sql"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">--可将多个小文件切片，合并为一个切片，进而由一个map任务处理</span></span><br><span class="line"><span class="keyword">set</span> hive.input.format<span class="operator">=</span>org.apache.hadoop.hive.ql.io.CombineHiveInputFormat</span><br></pre></td></tr></table></figure>

<h2 id="二、Reduce输出文件合并"><a href="#二、Reduce输出文件合并" class="headerlink" title="二、Reduce输出文件合并"></a>二、Reduce输出文件合并</h2><p>合并 Reduce 端输出的小文件，是指将多个小文件合并成大文件。目的是减少 HDFS 小文件数量。其原理是根据计算任务输出文件的平均大小进行判断，若符合条件，则单独启动一个额外的任务进行合并。</p>
<figure class="highlight sql"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">--开启合并map only任务输出的小文件</span></span><br><span class="line"><span class="keyword">set</span> hive.merge.mapfiles<span class="operator">=</span><span class="literal">true</span>;</span><br><span class="line"></span><br><span class="line"><span class="comment">--开启合并map reduce任务输出的小文件</span></span><br><span class="line"><span class="keyword">set</span> hive.merge.mapredfiles<span class="operator">=</span><span class="literal">true</span>;</span><br><span class="line"></span><br><span class="line"><span class="comment">--合并后的文件大小</span></span><br><span class="line"><span class="keyword">set</span> hive.merge.size.per.task<span class="operator">=</span><span class="number">256000000</span>;</span><br><span class="line"></span><br><span class="line"><span class="comment">--触发小文件合并任务的阈值，若某计算任务输出的文件平均大小低于该值，则触发合并</span></span><br><span class="line"><span class="keyword">set</span> hive.merge.smallfiles.avgsize<span class="operator">=</span><span class="number">16000000</span>;</span><br></pre></td></tr></table></figure>

<h1 id="其他优化"><a href="#其他优化" class="headerlink" title="其他优化"></a>其他优化</h1><h2 id="一、Fetch-抓取"><a href="#一、Fetch-抓取" class="headerlink" title="一、Fetch 抓取"></a>一、Fetch 抓取</h2><p>Fetc h抓取是指，Hive 中对某些情况的查询可以不必使用 MapReduce 计算。例如：select * from emp;在这种情况下，Hive 可以简单地读取 emp 对应的存储目录下的文件，然后输出查询结果到控制台。</p>
<figure class="highlight sql"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">--是否在特定场景转换为fetch 任务</span></span><br><span class="line"><span class="comment">--设置为none表示不转换</span></span><br><span class="line"><span class="comment">--设置为minimal表示支持select *，分区字段过滤，Limit等</span></span><br><span class="line"><span class="comment">--设置为more表示支持select 任意字段,包括函数，过滤，和limit等</span></span><br><span class="line"><span class="keyword">set</span> hive.fetch.task.conversion<span class="operator">=</span>more;</span><br></pre></td></tr></table></figure>

<h2 id="二、本地模式"><a href="#二、本地模式" class="headerlink" title="二、本地模式"></a>二、本地模式</h2><p>大多数的 Hadoop Job 是需要 Hadoop 提供的完整的可扩展性来处理大数据集的。不过，有时 Hive 的输入数据量是非常小的。在这种情况下，为查询触发执行任务消耗的时间可能会比实际 job 的执行时间要多的多。对于大多数这种情况，Hive 可以通过本地模式在单台机器上处理所有的任务。对于小数据集，执行时间可以明显被缩短。</p>
<figure class="highlight sql"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">--开启自动转换为本地模式</span></span><br><span class="line"><span class="keyword">set</span> hive.exec.mode.local.auto<span class="operator">=</span><span class="literal">true</span>;  </span><br><span class="line"></span><br><span class="line"><span class="comment">--设置local MapReduce的最大输入数据量，当输入数据量小于这个值时采用local  MapReduce的方式，默认为134217728，即128M</span></span><br><span class="line"><span class="keyword">set</span> hive.exec.mode.local.auto.inputbytes.max<span class="operator">=</span><span class="number">50000000</span>;</span><br><span class="line"></span><br><span class="line"><span class="comment">--设置local MapReduce的最大输入文件个数，当输入文件个数小于这个值时采用local MapReduce的方式，默认为4</span></span><br><span class="line"><span class="keyword">set</span> hive.exec.mode.local.auto.input.files.max<span class="operator">=</span><span class="number">10</span>;</span><br></pre></td></tr></table></figure>

<h3 id="三、并行执行"><a href="#三、并行执行" class="headerlink" title="三、并行执行"></a>三、并行执行</h3><p>Hive 会将一个 SQL 语句转化成一个或者多个 Stage，每个 Stage对应一个 MR Job。</p>
<p>默认情况下，Hive 同时只会执行一个 Stage。但是某 SQL 语句可能会包含多个 Stage，但这多个 Stage 可能并非完全互相依赖，也就是说有些 Stage 是可以并行执行的。</p>
<figure class="highlight sql"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">--启用并行执行优化</span></span><br><span class="line"><span class="keyword">set</span> hive.exec.parallel<span class="operator">=</span><span class="literal">true</span>;       </span><br><span class="line">    </span><br><span class="line"><span class="comment">--同一个sql允许最大并行度，默认为8</span></span><br><span class="line"><span class="keyword">set</span> hive.exec.parallel.thread.number<span class="operator">=</span><span class="number">8</span>;</span><br></pre></td></tr></table></figure>





























</article><div class="post-copyright"><div class="post-copyright__author"><span class="post-copyright-meta">文章作者: </span><span class="post-copyright-info"><a href="mailto:undefined">Oct25-X</a></span></div><div class="post-copyright__type"><span class="post-copyright-meta">文章链接: </span><span class="post-copyright-info"><a href="https://oct25-x.atomgit.net/oct25-xxxxx/2023/07/23/17%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8B%E6%95%B0%E6%8D%AE%E4%BB%93%E5%BA%93Hive05_%E4%BC%81%E4%B8%9A%E7%BA%A7%E8%B0%83%E4%BC%98/">https://oct25-x.atomgit.net/oct25-xxxxx/2023/07/23/17%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8B%E6%95%B0%E6%8D%AE%E4%BB%93%E5%BA%93Hive05_%E4%BC%81%E4%B8%9A%E7%BA%A7%E8%B0%83%E4%BC%98/</a></span></div><div class="post-copyright__notice"><span class="post-copyright-meta">版权声明: </span><span class="post-copyright-info">本博客所有文章除特别声明外，均采用 <a href="https://creativecommons.org/licenses/by-nc-sa/4.0/" target="_blank">CC BY-NC-SA 4.0</a> 许可协议。转载请注明来自 <a href="https://oct25-x.atomgit.net/oct25-xxxxx" target="_blank">Oct25-X</a>！</span></div></div><div class="tag_share"><div class="post-meta__tag-list"><a class="post-meta__tags" href="/oct25-xxxxx/tags/Hive/">Hive</a></div><div class="post_share"><div class="social-share" data-image="/oct25-xxxxx/img/cover_img/ctt17.png" data-sites="facebook,twitter,wechat,weibo,qq"></div><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/social-share.js/dist/css/share.min.css" media="print" onload="this.media='all'"><script src="https://cdn.jsdelivr.net/npm/social-share.js/dist/js/social-share.min.js" defer></script></div></div><nav class="pagination-post" id="pagination"><div class="prev-post pull-left"><a href="/oct25-xxxxx/2023/08/01/18SpringCloud01_%E6%9C%8D%E5%8A%A1%E6%B3%A8%E5%86%8C%E5%8F%91%E7%8E%B0%E4%B8%AD%E5%BF%83%20Consul/"><img class="prev-cover" src="/oct25-xxxxx/img/cover_img/ctt18.png" onerror="onerror=null;src='/oct25-xxxxx/img/404.jpg'" alt="cover of previous post"><div class="pagination-info"><div class="label">上一篇</div><div class="prev_info">SpringCloud01_服务注册发现中心 Consul</div></div></a></div><div class="next-post pull-right"><a href="/oct25-xxxxx/2023/07/22/16%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8B%E6%95%B0%E6%8D%AE%E4%BB%93%E5%BA%93Hive04_%E5%88%86%E5%8C%BA%E3%80%81%E5%88%86%E6%A1%B6%E8%A1%A8%E4%B8%8E%E5%8E%8B%E7%BC%A9/"><img class="next-cover" src="/oct25-xxxxx/img/cover_img/ctt16.png" onerror="onerror=null;src='/oct25-xxxxx/img/404.jpg'" alt="cover of next post"><div class="pagination-info"><div class="label">下一篇</div><div class="next_info">大数据之数据仓库Hive04_分区/分桶表与压缩</div></div></a></div></nav><div class="relatedPosts"><div class="headline"><i class="fas fa-thumbs-up fa-fw"></i><span>相关推荐</span></div><div class="relatedPosts-list"><div><a href="/oct25-xxxxx/2023/07/16/14%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8B%E6%95%B0%E6%8D%AE%E4%BB%93%E5%BA%93Hive02_DDL%20%E5%92%8C%20DML/" title="大数据之数据仓库Hive02_DDL 与 DML"><img class="cover" src="/oct25-xxxxx/img/cover_img/ctt14.png" alt="cover"><div class="content is-center"><div class="date"><i class="far fa-calendar-alt fa-fw"></i> 2023-07-16</div><div class="title">大数据之数据仓库Hive02_DDL 与 DML</div></div></a></div><div><a href="/oct25-xxxxx/2023/07/15/13%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8B%E6%95%B0%E6%8D%AE%E4%BB%93%E5%BA%93Hive01_%E5%9F%BA%E7%A1%80/" title="大数据之数据仓库Hive01_基础"><img class="cover" src="/oct25-xxxxx/img/cover_img/ctt13.png" alt="cover"><div class="content is-center"><div class="date"><i class="far fa-calendar-alt fa-fw"></i> 2023-07-15</div><div class="title">大数据之数据仓库Hive01_基础</div></div></a></div><div><a href="/oct25-xxxxx/2023/07/18/15%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8B%E6%95%B0%E6%8D%AE%E4%BB%93%E5%BA%93Hive03_%E6%9F%A5%E8%AF%A2%E5%92%8C%20Hive%20%E5%87%BD%E6%95%B0/" title="大数据之数据仓库Hive03_查询和 Hive 函数"><img class="cover" src="/oct25-xxxxx/img/cover_img/ctt15.png" alt="cover"><div class="content is-center"><div class="date"><i class="far fa-calendar-alt fa-fw"></i> 2023-07-18</div><div class="title">大数据之数据仓库Hive03_查询和 Hive 函数</div></div></a></div><div><a href="/oct25-xxxxx/2023/07/22/16%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8B%E6%95%B0%E6%8D%AE%E4%BB%93%E5%BA%93Hive04_%E5%88%86%E5%8C%BA%E3%80%81%E5%88%86%E6%A1%B6%E8%A1%A8%E4%B8%8E%E5%8E%8B%E7%BC%A9/" title="大数据之数据仓库Hive04_分区/分桶表与压缩"><img class="cover" src="/oct25-xxxxx/img/cover_img/ctt16.png" alt="cover"><div class="content is-center"><div class="date"><i class="far fa-calendar-alt fa-fw"></i> 2023-07-22</div><div class="title">大数据之数据仓库Hive04_分区/分桶表与压缩</div></div></a></div></div></div></div><div class="aside-content" id="aside-content"><div class="card-widget card-info"><div class="is-center"><div class="avatar-img"><img src="/oct25-xxxxx/img/avatar.jpg" onerror="this.onerror=null;this.src='/oct25-xxxxx/img/friend_404.gif'" alt="avatar"/></div><div class="author-info__name">Oct25-X</div><div class="author-info__description"></div></div><div class="card-info-data is-center"><div class="card-info-data-item"><a href="/oct25-xxxxx/archives/"><div class="headline">文章</div><div class="length-num">52</div></a></div><div class="card-info-data-item"><a href="/oct25-xxxxx/tags/"><div class="headline">标签</div><div class="length-num">17</div></a></div><div class="card-info-data-item"><a href="/oct25-xxxxx/categories/"><div class="headline">分类</div><div class="length-num">8</div></a></div></div><a id="card-info-btn" target="_blank" rel="noopener" href="https://github.com/xxxxxx"><i class="fab fa-github"></i><span>Follow Me</span></a></div><div class="sticky_layout"><div class="card-widget" id="card-toc"><div class="item-headline"><i class="fas fa-stream"></i><span>目录</span><span class="toc-percentage"></span></div><div class="toc-content"><ol class="toc"><li class="toc-item toc-level-1"><a class="toc-link" href="#%E8%AE%A1%E7%AE%97%E8%B5%84%E6%BA%90%E9%85%8D%E7%BD%AE"><span class="toc-number">1.</span> <span class="toc-text">计算资源配置</span></a><ol class="toc-child"><li class="toc-item toc-level-2"><a class="toc-link" href="#%E4%B8%80%E3%80%81Yarn-%E8%B5%84%E6%BA%90%E9%85%8D%E7%BD%AE"><span class="toc-number">1.1.</span> <span class="toc-text">一、Yarn 资源配置</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#%E4%BA%8C%E3%80%81MapReduce-%E8%B5%84%E6%BA%90%E9%85%8D%E7%BD%AE"><span class="toc-number">1.2.</span> <span class="toc-text">二、MapReduce 资源配置</span></a></li></ol></li><li class="toc-item toc-level-1"><a class="toc-link" href="#Explain-%E6%89%A7%E8%A1%8C%E8%AE%A1%E5%88%92"><span class="toc-number">2.</span> <span class="toc-text">Explain 执行计划</span></a><ol class="toc-child"><li class="toc-item toc-level-2"><a class="toc-link" href="#%E4%B8%80%E3%80%81Explain-%E6%A6%82%E8%BF%B0"><span class="toc-number">2.1.</span> <span class="toc-text">一、Explain 概述</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#%E4%BA%8C%E3%80%81%E5%9F%BA%E6%9C%AC%E8%AF%AD%E6%B3%95"><span class="toc-number">2.2.</span> <span class="toc-text">二、基本语法</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#%E4%B8%89%E3%80%81%E7%A4%BA%E4%BE%8B"><span class="toc-number">2.3.</span> <span class="toc-text">三、示例</span></a></li></ol></li><li class="toc-item toc-level-1"><a class="toc-link" href="#HQL-%E8%AF%AD%E6%B3%95%E4%BC%98%E5%8C%96%E4%B9%8B%E5%88%86%E7%BB%84%E8%81%9A%E5%90%88%E4%BC%98%E5%8C%96"><span class="toc-number">3.</span> <span class="toc-text">HQL 语法优化之分组聚合优化</span></a><ol class="toc-child"><li class="toc-item toc-level-2"><a class="toc-link" href="#%E4%B8%80%E3%80%81%E4%BC%98%E5%8C%96%E8%AF%B4%E6%98%8E"><span class="toc-number">3.1.</span> <span class="toc-text">一、优化说明</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#%E4%BA%8C%E3%80%81%E4%BC%98%E5%8C%96%E7%A4%BA%E4%BE%8B"><span class="toc-number">3.2.</span> <span class="toc-text">二、优化示例</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#%E4%BC%98%E5%8C%96%E5%89%8D%E6%89%A7%E8%A1%8C%E8%AE%A1%E5%88%92"><span class="toc-number">3.2.1.</span> <span class="toc-text">优化前执行计划</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E4%BC%98%E5%8C%96%E5%90%8E%E6%89%A7%E8%A1%8C%E8%AE%A1%E5%88%92"><span class="toc-number">3.2.2.</span> <span class="toc-text">优化后执行计划</span></a></li></ol></li></ol></li><li class="toc-item toc-level-1"><a class="toc-link" href="#HQL-%E8%AF%AD%E6%B3%95%E4%BC%98%E5%8C%96%E4%B9%8B-Join-%E4%BC%98%E5%8C%96"><span class="toc-number">4.</span> <span class="toc-text">HQL 语法优化之 Join 优化</span></a><ol class="toc-child"><li class="toc-item toc-level-2"><a class="toc-link" href="#%E4%B8%80%E3%80%81join-%E7%AE%97%E6%B3%95%E8%AF%B4%E6%98%8E"><span class="toc-number">4.1.</span> <span class="toc-text">一、join 算法说明</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#Common-Join"><span class="toc-number">4.1.1.</span> <span class="toc-text">Common Join</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#Map-Join"><span class="toc-number">4.1.2.</span> <span class="toc-text">Map Join</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#Bucket-Map-Join"><span class="toc-number">4.1.3.</span> <span class="toc-text">Bucket Map Join</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#Sort-Merge-Bucket-Map-Join"><span class="toc-number">4.1.4.</span> <span class="toc-text">Sort Merge Bucket Map Join</span></a></li></ol></li><li class="toc-item toc-level-2"><a class="toc-link" href="#%E4%BA%8C%E3%80%81Map-Join"><span class="toc-number">4.2.</span> <span class="toc-text">二、Map Join</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#%E4%BC%98%E5%8C%96%E5%8F%82%E6%95%B0"><span class="toc-number">4.2.1.</span> <span class="toc-text">优化参数</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E4%BC%98%E5%8C%96%E7%A4%BA%E4%BE%8B"><span class="toc-number">4.2.2.</span> <span class="toc-text">优化示例</span></a><ol class="toc-child"><li class="toc-item toc-level-4"><a class="toc-link" href="#%E4%BC%98%E5%8C%96%E5%89%8D%E6%89%A7%E8%A1%8C%E8%AE%A1%E5%88%92-1"><span class="toc-number">4.2.2.1.</span> <span class="toc-text">优化前执行计划</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#%E6%96%B9%E6%A1%88%E4%B8%80%E4%BC%98%E5%8C%96%E5%90%8E%E6%89%A7%E8%A1%8C%E8%AE%A1%E5%88%92"><span class="toc-number">4.2.2.2.</span> <span class="toc-text">方案一优化后执行计划</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#%E6%96%B9%E6%A1%88%E4%BA%8C%E4%BC%98%E5%8C%96%E5%90%8E%E6%89%A7%E8%A1%8C%E8%AE%A1%E5%88%92"><span class="toc-number">4.2.2.3.</span> <span class="toc-text">方案二优化后执行计划</span></a></li></ol></li></ol></li></ol></li><li class="toc-item toc-level-1"><a class="toc-link" href="#HQL-%E8%AF%AD%E6%B3%95%E4%BC%98%E5%8C%96%E4%B9%8B%E6%95%B0%E6%8D%AE%E5%80%BE%E6%96%9C"><span class="toc-number">5.</span> <span class="toc-text">HQL 语法优化之数据倾斜</span></a><ol class="toc-child"><li class="toc-item toc-level-2"><a class="toc-link" href="#%E4%B8%80%E3%80%81%E6%95%B0%E6%8D%AE%E5%80%BE%E6%96%9C%E6%A6%82%E8%BF%B0"><span class="toc-number">5.1.</span> <span class="toc-text">一、数据倾斜概述</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#%E4%BA%8C%E3%80%81%E5%88%86%E7%BB%84%E8%81%9A%E5%90%88%E5%AF%BC%E8%87%B4%E6%95%B0%E6%8D%AE%E5%80%BE%E6%96%9C"><span class="toc-number">5.2.</span> <span class="toc-text">二、分组聚合导致数据倾斜</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#Map-Side-%E8%81%9A%E5%90%88"><span class="toc-number">5.2.1.</span> <span class="toc-text">Map-Side 聚合</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#Skew-GroupBy-%E4%BC%98%E5%8C%96"><span class="toc-number">5.2.2.</span> <span class="toc-text">Skew-GroupBy 优化</span></a></li></ol></li></ol></li><li class="toc-item toc-level-1"><a class="toc-link" href="#HQL-%E8%AF%AD%E6%B3%95%E4%BC%98%E5%8C%96%E4%B9%8B%E5%B0%8F%E6%96%87%E4%BB%B6%E5%90%88%E5%B9%B6"><span class="toc-number">6.</span> <span class="toc-text">HQL 语法优化之小文件合并</span></a><ol class="toc-child"><li class="toc-item toc-level-2"><a class="toc-link" href="#%E4%B8%80%E3%80%81Map-%E7%AB%AF%E8%BE%93%E5%85%A5%E6%96%87%E4%BB%B6%E5%90%88%E5%B9%B6"><span class="toc-number">6.1.</span> <span class="toc-text">一、Map 端输入文件合并</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#%E4%BA%8C%E3%80%81Reduce%E8%BE%93%E5%87%BA%E6%96%87%E4%BB%B6%E5%90%88%E5%B9%B6"><span class="toc-number">6.2.</span> <span class="toc-text">二、Reduce输出文件合并</span></a></li></ol></li><li class="toc-item toc-level-1"><a class="toc-link" href="#%E5%85%B6%E4%BB%96%E4%BC%98%E5%8C%96"><span class="toc-number">7.</span> <span class="toc-text">其他优化</span></a><ol class="toc-child"><li class="toc-item toc-level-2"><a class="toc-link" href="#%E4%B8%80%E3%80%81Fetch-%E6%8A%93%E5%8F%96"><span class="toc-number">7.1.</span> <span class="toc-text">一、Fetch 抓取</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#%E4%BA%8C%E3%80%81%E6%9C%AC%E5%9C%B0%E6%A8%A1%E5%BC%8F"><span class="toc-number">7.2.</span> <span class="toc-text">二、本地模式</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#%E4%B8%89%E3%80%81%E5%B9%B6%E8%A1%8C%E6%89%A7%E8%A1%8C"><span class="toc-number">7.2.1.</span> <span class="toc-text">三、并行执行</span></a></li></ol></li></ol></li></ol></div></div><div class="card-widget card-recent-post"><div class="item-headline"><i class="fas fa-history"></i><span>最新文章</span></div><div class="aside-list"><div class="aside-list-item"><a class="thumbnail" href="/oct25-xxxxx/2024/02/24/52%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8B%E5%A4%9A%E8%8C%83%E5%BC%8F%E7%BC%96%E7%A8%8B%E8%AF%AD%E8%A8%80Scala09_%E9%9A%90%E5%BC%8F%E8%BD%AC%E6%8D%A2%E3%80%81%E5%BC%82%E5%B8%B8%E5%92%8C%E6%B3%9B%E5%9E%8B/" title="大数据之多范式编程语言 Scala09_隐式转换、异常和泛型"><img src="/oct25-xxxxx/img/cover_img/ctt52.png" onerror="this.onerror=null;this.src='/oct25-xxxxx/img/404.jpg'" alt="大数据之多范式编程语言 Scala09_隐式转换、异常和泛型"/></a><div class="content"><a class="title" href="/oct25-xxxxx/2024/02/24/52%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8B%E5%A4%9A%E8%8C%83%E5%BC%8F%E7%BC%96%E7%A8%8B%E8%AF%AD%E8%A8%80Scala09_%E9%9A%90%E5%BC%8F%E8%BD%AC%E6%8D%A2%E3%80%81%E5%BC%82%E5%B8%B8%E5%92%8C%E6%B3%9B%E5%9E%8B/" title="大数据之多范式编程语言 Scala09_隐式转换、异常和泛型">大数据之多范式编程语言 Scala09_隐式转换、异常和泛型</a><time datetime="2024-02-24T04:55:34.000Z" title="发表于 2024-02-24 12:55:34">2024-02-24</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/oct25-xxxxx/2024/02/24/51%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8B%E5%A4%9A%E8%8C%83%E5%BC%8F%E7%BC%96%E7%A8%8B%E8%AF%AD%E8%A8%80Scala08_%E6%A8%A1%E5%BC%8F%E5%8C%B9%E9%85%8D/" title="大数据之多范式编程语言 Scala08_模式匹配"><img src="/oct25-xxxxx/img/cover_img/ctt51.png" onerror="this.onerror=null;this.src='/oct25-xxxxx/img/404.jpg'" alt="大数据之多范式编程语言 Scala08_模式匹配"/></a><div class="content"><a class="title" href="/oct25-xxxxx/2024/02/24/51%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8B%E5%A4%9A%E8%8C%83%E5%BC%8F%E7%BC%96%E7%A8%8B%E8%AF%AD%E8%A8%80Scala08_%E6%A8%A1%E5%BC%8F%E5%8C%B9%E9%85%8D/" title="大数据之多范式编程语言 Scala08_模式匹配">大数据之多范式编程语言 Scala08_模式匹配</a><time datetime="2024-02-24T00:46:12.000Z" title="发表于 2024-02-24 08:46:12">2024-02-24</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/oct25-xxxxx/2024/02/22/50%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8B%E5%A4%9A%E8%8C%83%E5%BC%8F%E7%BC%96%E7%A8%8B%E8%AF%AD%E8%A8%80Scala07_%E9%9B%86%E5%90%88/" title="大数据之多范式编程语言 Scala07_集合"><img src="/oct25-xxxxx/img/cover_img/ctt50.png" onerror="this.onerror=null;this.src='/oct25-xxxxx/img/404.jpg'" alt="大数据之多范式编程语言 Scala07_集合"/></a><div class="content"><a class="title" href="/oct25-xxxxx/2024/02/22/50%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8B%E5%A4%9A%E8%8C%83%E5%BC%8F%E7%BC%96%E7%A8%8B%E8%AF%AD%E8%A8%80Scala07_%E9%9B%86%E5%90%88/" title="大数据之多范式编程语言 Scala07_集合">大数据之多范式编程语言 Scala07_集合</a><time datetime="2024-02-22T08:16:10.000Z" title="发表于 2024-02-22 16:16:10">2024-02-22</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/oct25-xxxxx/2024/02/21/49%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8B%E5%A4%9A%E8%8C%83%E5%BC%8F%E7%BC%96%E7%A8%8B%E8%AF%AD%E8%A8%80Scala06_%E9%9D%A2%E5%90%91%E5%AF%B9%E8%B1%A1/" title="大数据之多范式编程语言 Scala06_面向对象"><img src="/oct25-xxxxx/img/cover_img/ctt49.png" onerror="this.onerror=null;this.src='/oct25-xxxxx/img/404.jpg'" alt="大数据之多范式编程语言 Scala06_面向对象"/></a><div class="content"><a class="title" href="/oct25-xxxxx/2024/02/21/49%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8B%E5%A4%9A%E8%8C%83%E5%BC%8F%E7%BC%96%E7%A8%8B%E8%AF%AD%E8%A8%80Scala06_%E9%9D%A2%E5%90%91%E5%AF%B9%E8%B1%A1/" title="大数据之多范式编程语言 Scala06_面向对象">大数据之多范式编程语言 Scala06_面向对象</a><time datetime="2024-02-21T07:01:10.000Z" title="发表于 2024-02-21 15:01:10">2024-02-21</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/oct25-xxxxx/2024/02/20/48%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8B%E5%A4%9A%E8%8C%83%E5%BC%8F%E7%BC%96%E7%A8%8B%E8%AF%AD%E8%A8%80Scala05_%E5%87%BD%E6%95%B0%E5%BC%8F%E7%BC%96%E7%A8%8B/" title="大数据之多范式编程语言 Scala05_函数式编程"><img src="/oct25-xxxxx/img/cover_img/ctt48.png" onerror="this.onerror=null;this.src='/oct25-xxxxx/img/404.jpg'" alt="大数据之多范式编程语言 Scala05_函数式编程"/></a><div class="content"><a class="title" href="/oct25-xxxxx/2024/02/20/48%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8B%E5%A4%9A%E8%8C%83%E5%BC%8F%E7%BC%96%E7%A8%8B%E8%AF%AD%E8%A8%80Scala05_%E5%87%BD%E6%95%B0%E5%BC%8F%E7%BC%96%E7%A8%8B/" title="大数据之多范式编程语言 Scala05_函数式编程">大数据之多范式编程语言 Scala05_函数式编程</a><time datetime="2024-02-20T12:05:11.000Z" title="发表于 2024-02-20 20:05:11">2024-02-20</time></div></div></div></div></div></div></main><footer id="footer" style="background-image: url('/oct25-xxxxx/img/top_img/top_bg16.jpg')"><div id="footer-wrap"><div class="copyright">&copy;2020 - 2024 By Oct25-X</div><div class="framework-info"><span>框架 </span><a target="_blank" rel="noopener" href="https://hexo.io">Hexo</a><span class="footer-separator">|</span><span>主题 </span><a target="_blank" rel="noopener" href="https://github.com/jerryc127/hexo-theme-butterfly">Butterfly</a></div></div></footer></div><div id="rightside"><div id="rightside-config-hide"><button id="readmode" type="button" title="阅读模式"><i class="fas fa-book-open"></i></button><button id="translateLink" type="button" title="简繁转换">繁</button><button id="darkmode" type="button" title="浅色和深色模式转换"><i class="fas fa-adjust"></i></button><button id="hide-aside-btn" type="button" title="单栏和双栏切换"><i class="fas fa-arrows-alt-h"></i></button></div><div id="rightside-config-show"><button id="rightside_config" type="button" title="设置"><i class="fas fa-cog fa-spin"></i></button><button class="close" id="mobile-toc-button" type="button" title="目录"><i class="fas fa-list-ul"></i></button><button id="go-up" type="button" title="回到顶部"><i class="fas fa-arrow-up"></i></button></div></div><div id="local-search"><div class="search-dialog"><nav class="search-nav"><span class="search-dialog-title">本地搜索</span><span id="loading-status"></span><button class="search-close-button"><i class="fas fa-times"></i></button></nav><div class="is-center" id="loading-database"><i class="fas fa-spinner fa-pulse"></i><span>  数据库加载中</span></div><div class="search-wrap"><div id="local-search-input"><div class="local-search-box"><input class="local-search-box--input" placeholder="搜索文章" type="text"/></div></div><hr/><div id="local-search-results"></div></div></div><div id="search-mask"></div></div><div id="rightMenu"><div class="rightMenu-group rightMenu-small"><div class="rightMenu-item" id="menu-backward"><i class="fa-solid fa-arrow-left"></i></div><div class="rightMenu-item" id="menu-forward"><i class="fa-solid fa-arrow-right"></i></div><div class="rightMenu-item" id="menu-refresh"><i class="fa-solid fa-arrow-rotate-right"></i></div><div class="rightMenu-item" id="menu-home"><i class="fa-solid fa-house"></i></div></div><div class="rightMenu-group rightMenu-line rightMenuOther"><a class="rightMenu-item menu-link" href="/archives/"><i class="fa-solid fa-archive"></i><span>文章归档</span></a><a class="rightMenu-item menu-link" href="/categories/"><i class="fa-solid fa-folder-open"></i><span>文章分类</span></a><a class="rightMenu-item menu-link" href="/tags/"><i class="fa-solid fa-tags"></i><span>文章标签</span></a></div><div class="rightMenu-group rightMenu-line rightMenuNormal"><a class="rightMenu-item" id="menu-translate"><i class="fa-solid fa-earth-asia"></i><span>繁简切换</span></a><div class="rightMenu-item" id="menu-darkmode"><i class="fa-solid fa-moon"></i><span>切换模式</span></div></div></div><div id="rightmenu-mask"></div><div><script src="/oct25-xxxxx/js/utils.js"></script><script src="/oct25-xxxxx/js/main.js"></script><script src="/oct25-xxxxx/js/tw_cn.js"></script><script src="https://cdn.jsdelivr.net/npm/@fancyapps/ui/dist/fancybox.umd.js"></script><script src="/oct25-xxxxx/js/search/local-search.js"></script><div class="js-pjax"></div><script defer src="https://npm.elemecdn.com/jquery@latest/dist/jquery.min.js"></script><script defer data-pjax src="/js/rightMenu.js"></script><script defer="defer" id="ribbon" src="https://cdn.jsdelivr.net/npm/butterfly-extsrc@1/dist/canvas-ribbon.min.js" size="150" alpha="0.6" zIndex="-1" mobile="false" data-click="true"></script><script src="https://cdn.jsdelivr.net/npm/butterfly-extsrc@1/dist/activate-power-mode.min.js"></script><script>POWERMODE.colorful = true;
POWERMODE.shake = true;
POWERMODE.mobile = false;
document.body.addEventListener('input', POWERMODE);
</script><script id="click-heart" src="https://cdn.jsdelivr.net/npm/butterfly-extsrc@1/dist/click-heart.min.js" async="async" mobile="false"></script><script async data-pjax src="//busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js"></script></div></body></html>